from selenium import webdriver
from selenium.webdriver import ChromeOptions
from time import sleep
from lxml import etree
import csv

"""
#selenium.webdriver.chrome.options.Options
用于设置chrome浏览器，目前使用过的参数：
1、阻止chrome弹窗的出现：'profile.default_content_setting_values' :{'notifications' : 2}
2、不加载图片："profile.managed_default_content_settings.images":2
3、无头浏览器，--headless
"""

class Vip_Product_Spider():
    def __init__(self):
        self.keyword=input("请输入采集关键字：")
        self.start_page=input("请输入起始页数：")
        self.end_page = input("请输入结束页数：")

        # 配置chrome浏览器的相关参数
        option = ChromeOptions()
        option.add_experimental_option('prefs', {'profile.managed_default_content_settings.images': 2})
        option.add_argument('--headless')
        option.add_argument('--disable-gpu')  # 设置无头浏览器
        self.bro=webdriver.Chrome(options=option)
        # Chrome DevTools Protocol ：Evaluates given script in every frame upon creation (before loading frame's scripts).
        # 目的在打开窗口前隐藏window.navigator.webdriver
        self.bro.execute_cdp_cmd("Page.addScriptToEvaluateOnNewDocument", {
            "source": """
                Object.defineProperty(navigator, 'webdriver', {
                  get: () => undefined
                })
              """
        })  # 可能失效

    def main(self):
        for i in range(int(self.start_page),int(self.end_page)+1):
            sleep(3)
            url="https://category.vip.com/suggest.php?keyword=%s&page=%d"%(self.keyword,i)
            self.load_data(url)
        self.bro.quit()

    def load_data(self,url):
        self.bro.get(url)
        self.bro.execute_script('window.scrollTo(0, document.body.scrollHeight)')  # 向下拉动一屏
        self.bro.execute_script('window.scrollTo(0, document.body.scrollHeight)')  # 向下拉动一屏
        sleep(5)
        self.parser_data()

    def parser_data(self):
        # 通过etree解析页面；
        html=etree.HTML(self.bro.page_source)
        div_list=html.xpath('//section[@class="goods-list c-goods-list--normal"]/div')[1:]
        for div in div_list:
            sleep(0.5)
            dic={}
            try:
                dic["title"]=div.xpath('.//div[@class="c-goods-item__name  c-goods-item__name--two-line"]/text()')[0]
            except:
                dic["title"]=""
            try:
                dic["sale_price"]=div.xpath('.//div[@class="c-goods-item__sale-price J-goods-item__sale-price"]//text()')[1]
            except:
                dic["sale_price"]=""
            try:
                dic["market_price"]=div.xpath('.//div[@class="c-goods-item__market-price  J-goods-item__market-price"]//text()')[1]
            except:
                dic["market_price"]=""
            try:
                dic["discount"]=div.xpath('.//div[@class="c-goods-item__discount  J-goods-item__discount"]/text()')[0]
            except:
                dic["discount"]=""
            try:
                dic["img_link"]="http:"+div.xpath('.//img[@class="J-goods-item__img"]/@src')[0]
            except:
                dic["img_link"]=""
            try:
                dic["details_link"]="https:"+div.xpath('.//a[@target="_blank"]/@href')[0]
            except:
                dic["details_link"]=""
            with open(".//vip-women-brand.csv", "a", encoding="utf-8") as f:
                writer = csv.DictWriter(f, dic.keys())
                writer.writerow(dic)


if __name__ == "__main__":
    Spider=Vip_Product_Spider()
    Spider.main()